#!/usr/bin/python
# -*- coding: utf-8 -*-

import re
import time
import Image
import urllib
import MySQLdb
import StringIO
import datetime
import socket
import hashlib

socket.setdefaulttimeout(10)

#score the data by using the stats of Youku
def Score(data):
    pv = data[3]
    totalcomment = data[4]
    fav = data[5]
    updown = data[6] - data[7]
    reference = data[8]
    return fav

def GetFormat(t):
    if t < 10:
    	return "0" + str(t)
    return str(t)

def FormatTime(second):
    hour = GetFormat(second / 3600)
    min = GetFormat(second % 3600 / 60)
    sec = GetFormat(second % 60)
    return hour + ":" + min + ":" + sec

def MyCmp(data1, data2):
    if Score(data1) > Score(data2):
    	return -1
    return 1
def MyCmpFav(data1, data2):
    if data1[5] > data2[5]:
    	return -1
    return 1
def MyCmpUpMinusDown(data1, data2):
    if data1[6]-data1[7] > data2[6]-data2[7]:
    	return -1
    return 1
def MyCmpUpDivideDown(data1, data2):
    if data1[7] == 0:
        temp1 = 1
    else:
        temp1 = data1[7]
    if data2[7] == 0:
        temp2 = 1
    else:
        temp2 = data2[7]
    if data1[6] * 1.0 /temp1 > data2[6] * 1.0 / temp2:
    	return -1
    return 1

def HisInter(h1, h2):
    up = 0
    down = 0
    down2 = 0
    for i in range(0, 768):
    	up = up + min(h1[i], h2[i])
    	down = down + h1[i]
    	down2 = down2 + h2[i]
    return min(1.0 * up / down, 1.0 * up / down2)

def Duplicate(thumb1, thumb2, duration1, duration2):
    if abs(duration2 - duration1) > 3:
    	return False
    return HisInter(thumb1.histogram(), thumb2.histogram()) > 0.8

def Deduplicate(l1, l2):
    thumbnails = []
    for i in range(0, video_cnt):
        try:
            data = urllib.urlopen(l1[i][2]).read()
            thumbnails.append(Image.open(StringIO.StringIO(data)))
        except:
            thumbnails.append('nothing')
            print 'image download error'
            l2.append(l1[i])
            continue
        label = True
        for j in range(0, i):
            try:
                if Duplicate(thumbnails[j], thumbnails[i], l1[j][11], l1[i][11]):
                    label = False
                    print j, i
                    break
            except:
                print 'image process error'
        if label:
            l2.append(l1[i])
    print len(thumbnails)

def DeduplicateMD5(l1, l2):
    md5s = {}
    for i in range(0, video_cnt):
        success = 0
        while success < 5:
            try:
                md5 = hashlib.md5(urllib.urlopen(l1[i][2]).read()).hexdigest()
                if not md5s.has_key(md5):
                    l2.append(l1[i])
                    md5s[md5] = 1
                break
            except Exception, data:
                print Exception, "XXX", data
                success = success + 1

def Normalize(data):
    pubdate = data[9]
    fav = data[5]
    up = data[6]
    down = data[7]
    timediff = cur_time -  time.mktime(pubdate.timetuple())
    if timediff <= 4 * 24 * 3600:
        data[5] = fav * 4.0 * 24 * 3600 / timediff
        data[6] = up * 4.0 * 24 * 3600 / timediff
        data[7] = down * 4.0 * 24 * 3600 / timediff

def weiboScore(data):
    return data[14]

def weiboMyCmp(data1, data2):
    if weiboScore(data1) > weiboScore(data2):
        return -1
    return 1
def renrenShareScore(data):
    return data[13]

def renrenShareCmp(data1, data2):
    if renrenShareScore(data1) > renrenShareScore(data2):
    	return -1
    return 1

def getFinalList(vlist1, vlist2, vlist3):
    vmap = {}
    index = 0
    video_cnt1 = len(vlist1)
    for i in vlist1:
        vmap[i[0]] = i
        vmap[i[0]][5] = (video_cnt1 - index) * 1.0 / video_cnt1
        index = index + 1
    index = 0
    video_cnt2 = len(vlist2)
    for i in vlist2:
        if i[0] not in vmap:
            vmap[i[0]] = i
            vmap[i[0]][5] = (video_cnt2 - index) * 1.0 / video_cnt2
        else:
            vmap[i[0]][5] = vmap[i[0]][5] + (video_cnt2 - index) * 1.0 / video_cnt2
        index = index + 1
    index = 0
    video_cnt3 = len(vlist3)
    for i in vlist3:
        if i[0] not in vmap:
            vmap[i[0]] = i
            vmap[i[0]][5] = (video_cnt3 - index) * 1.0 / video_cnt3
        else:
            vmap[i[0]][5] = vmap[i[0]][5] + (video_cnt3 - index) * 1.0 / video_cnt3
        index = index + 1
    vlist = vmap.values()
    vlist.sort(MyCmp)
    return vlist



#得到0-30天前的日期 (0,30)
ISOTIMEFORMAT = '%Y-%m-%d'
ISOTIMEFORMAT2 = '%Y-%m-%d %H:%M:%S'
cur_time = time.time()
from_time = cur_time - 30 * 24 * 3600;
from_time_str = time.strftime(ISOTIMEFORMAT, time.localtime(from_time))
to_time = cur_time;
to_time_str = time.strftime(ISOTIMEFORMAT, time.localtime(to_time))

#连接数据库
conn = MySQLdb.Connection("172.16.7.58", "root", "researchsucks", "X", 3306)
conn.set_character_set("utf8");
cur = conn.cursor()

#执行查询
#优酷排序
cur.execute("select id, title, imagelink,  pv, totalcomment, fav, up, down, reference, pubdate, playlink, duration, cats \
        from video_info where pubdate>=date('" + from_time_str + "') and pubdate<date('" + to_time_str + "')")

#基于流行度排序0-30天前视频
#fav
vlist1 = []
#up_minus_down
vlist2 = []
#up_divide_down
vlist3 = []
for data in cur.fetchall():
    data = list(data)
    Normalize(data)
    #fav
    if data[5] >= 30:
        vlist1.append(data)
    #up_minus_down
    if data[6] - data[7] >= 120:
        vlist2.append(data)
    #up_divide_down
    if data[7] == 0:
        temp = 1
    else:
        temp = data[7]
    if data[6] * 1.0 / temp >= 30:
        vlist3.append(data)

vlist1.sort(MyCmpFav)
vlist2.sort(MyCmpUpMinusDown)
vlist3.sort(MyCmpUpDivideDown)
print len(vlist1)
print len(vlist2)
print len(vlist3)
vmap = {}
index = 0
video_cnt1 = len(vlist1)
for i in vlist1:
    vmap[i[0]] = i
    vmap[i[0]][5] = (video_cnt1 - index) * 1.0 / video_cnt1
    index = index + 1
index = 0
video_cnt2 = len(vlist2)
for i in vlist2:
    if i[0] not in vmap:
        vmap[i[0]] = i
        vmap[i[0]][5] = (video_cnt2 - index) * 1.0 / video_cnt2
    else:
        vmap[i[0]][5] = vmap[i[0]][5] + (video_cnt2 - index) * 1.0 / video_cnt2
    index = index + 1
index = 0
video_cnt3 = len(vlist3)
for i in vlist3:
    if i[0] not in vmap:
        vmap[i[0]] = i
        vmap[i[0]][5] = (video_cnt3 - index) * 1.0 / video_cnt3
    else:
        vmap[i[0]][5] = vmap[i[0]][5] + (video_cnt3 - index) * 1.0 / video_cnt3
    index = index + 1
vlistyouku = vmap.values()
print len(vlistyouku)
vlistyouku.sort(MyCmp)


#weibo排序
cur.execute("select video_info.id, video_info.title, video_info.imagelink,  video_info.pv, video_info.totalcomment, video_info.fav, video_info.up, video_info.down, video_info.reference, video_info.pubdate, video_info.playlink, video_info.duration, video_info.cats, weibo_map.id, weibo_map.repostCount, weibo_map.url from weibo_map, video_info where weibo_map.videoPubdate>='2012-03-03' and weibo_map.videoPubdate>=date('" + from_time_str + "') and weibo_map.videoPubdate<date('" + to_time_str + "') and weibo_map.repostCount is not NULL and weibo_map.videoid=video_info.id")

vlisttemp = []
vlistweibo = []
for data in cur.fetchall():
    vlisttemp.append(data)

vlisttemp.sort(weiboMyCmp)
weiboset = set()

for i in vlisttemp:
    if i[14] < 1000:
        break
    if i[0] not in weiboset:
        vlistweibo.append([i[0], i[1], i[2], i[3], i[4], i[5], i[6], i[7], i[8], i[9], i[10], i[11], i[12]])
        weiboset.add(i[0])

print "weibo length:", len(vlistweibo)

#renren排序
cur.execute("select id, title, imagelink, pv, totalcomment, fav, up, down, reference, pubdate, playlink, duration, cats, renrenShare, renrenWatch, renrenUser, renrenVideo from video_info where renrenShare!=-1 and pubdate>='2012-03-19' and pubdate>=date('" + from_time_str + "') and pubdate<date('" + to_time_str + "') order by pubdate desc")

vlisttemp = []
vlistrenren = []
for data in cur.fetchall():
    data = list(data)
    vlisttemp.append(data)

vlisttemp.sort(renrenShareCmp)
for i in vlisttemp:
    if i[13] < 100:
        break
    vlistrenren.append([i[0], i[1], i[2], i[3], i[4], i[5], i[6], i[7], i[8], i[9], i[10], i[11], i[12]])


vlist = getFinalList(vlistyouku, vlistweibo, vlistrenren)
video_cnt = len(vlist)


#将top的视频信息写入xml, json
#output = open('/home/chaojiansong/videobook/notinsvn/pop_sorted.xml', 'w')
output = open('/home/chaojiansong/videobook/notinsvn/pop_sorted.xml', 'w')
output2 = open('/home/chaojiansong/videobook/notinsvn/pop_sorted.json', 'w')
output.write("<xml>\n")
output2.write("{\"images\":[\n")
output3 = open('/home/chaojiansong/videobook/notinsvn/thumbnail_urls', 'w')
output4 = open('/home/chaojiansong/videobook/notinsvn/video_score_list.uniq', 'w')

for i in range(0, video_cnt):
    output.write("<item>\n")
    output.write("<id>" + vlist[i][0] + "</id>\n")
    output.write("<title><![CDATA[" + vlist[i][1] + "]]></title>\n") 
    output.write("<thumbnail><![CDATA[" + vlist[i][2] + "]]></thumbnail>\n")
    output.write("<pv>" + str(vlist[i][3]) + "</pv>\n")
    output.write("<totalcomment>" + str(vlist[i][4]) + "</totalcomment>\n")
    output.write("<fav>" + str(vlist[i][5]) + "</fav>\n")
    output.write("<up>" + str(vlist[i][6]) + "</up>\n")
    output.write("<down>" + str(vlist[i][7]) + "</down>\n")
    output.write("<reference>" + str(vlist[i][8]) + "</reference>\n")
    output.write("<pubdate>" + str(vlist[i][9]) + "</pubdate>\n")
    output.write("<playlink><![CDATA[" + vlist[i][10] + "]]></playlink>\n")
    output.write("<duration>" + str(vlist[i][11]) + "</duration>\n")
    output.write("<cats>" + vlist[i][12] + "</cats>\n")
    output.write("</item>\n")
vlist2 = []
Deduplicate(vlist, vlist2)
#DeduplicateMD5(vlist, vlist2)

tot = len(vlist2)
print tot
for i in range(0, tot):
    output2.write("{\"src\":\"" + vlist2[i][2] + "\", \"title\":\"" + re.sub(r'"', '\\"', vlist2[i][1]) + "\", \"href\":\"" + \
            vlist2[i][0] + "\", " + "\"playlink\":\"" + vlist2[i][10] + "\", \"duration\":\"" + FormatTime(vlist2[i][11]) +  "\"}")
    output4.write(vlist2[i][0] + '\n' + str(vlist2[i][5]) + '\n' + vlist2[i][10] + '\n' + vlist2[i][2] + '\n' + vlist2[i][1] + '\n' + FormatTime(vlist2[i][11]) + '\n')
    if i != tot - 1:
    	output2.write(',')
    output2.write('\n')
    #将视频记入pop_history
    cur.execute("insert ignore into pop_history(video_id) values('" + vlist2[i][0] + "')")

#记录视频缩略图
for i in range(0, video_cnt):
    output3.write(vlist[i][2] + '\n')

#将user_system_rec_pos归位;
#cur.execute("update user_system_rec_pos set currentpos=1")
cur.execute("update user_system_rec_pos set popularcurrentpos=1, popularnextpos=1, personalcurrentpos=1, personalnextpos=1")

output.write("</xml>\n")
output2.write("]}\n")
output.close()
output2.close()
output3.close()
output4.close()

cur.close()
conn.close()
